{
 "cells": [
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "import os\n",
    "import json\n",
    "import random\n",
    "from datasets import load_dataset,concatenate_datasets,Dataset\n",
    "from tqdm import tqdm"
   ],
   "id": "3f1f041d03be5687"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "os.listdir(\"qa_data\")",
   "id": "2d336a5f5f968528"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "datas=[]\n",
    "for f in os.listdir(\"qa_data\"):\n",
    "    ds=load_dataset('json', data_files=f'qa_data/{f}',field='data')\n",
    "    datas.append(ds)"
   ],
   "id": "d2bac498ee42d8c0"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "datas",
   "id": "9e1d947da4ed7131"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "datasets = concatenate_datasets([ds['train'] for ds in datas])",
   "id": "d8382077f2fd11ff"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "datasets",
   "id": "d23551eba06eef3d"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "datasets[5070]",
   "id": "8383e3d023c1065"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "datasets[52270]",
   "id": "65c57567cf90d17f"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "prompts_up=[\"请根据下文回答这个问题：{}\\n\\n\\n 以下是文章内容： {}\",\n",
    "        \"你现在是一个阅读理解的高手，请根据文中的内容回答：{}\\n\\n\\n 内容如下：{}\",\n",
    "        \"利用文中的信息回答问题：{}\\n\\n\\n 原文：{}\",\n",
    "        \"利用下面文章回答：{}\\n\\n\\n 文章：{}\"]\n",
    "prompts_down=[\"{} \\n\\n\\n请根据上文回答这个问题：{}\",\n",
    "        \"{} \\n\\n\\n你现在是一个阅读理解的高手，请根据文中的内容回答：{}\",\n",
    "        \"{}\\n\\n\\n利用文中的信息回答问题：{}\",\n",
    "        \"{}\\n\\n\\n利用这段文字回答：{}\"]"
   ],
   "id": "847f1ba5b5e09067"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "inputs,outputs=[],[]\n",
    "for data in tqdm(datasets):\n",
    "    try:\n",
    "        context=data['paragraphs'][0]['context']\n",
    "        answer=data['paragraphs'][0]['qas'][0]['answers'][0]['text']\n",
    "        question=data['paragraphs'][0]['qas'][0]['question']\n",
    "        if random.random()>=0.5:\n",
    "            prompt=random.choice(prompts_up)\n",
    "            instruction=prompt.format(question,context)\n",
    "        else:\n",
    "            prompt=random.choice(prompts_down)\n",
    "            instruction=prompt.format(context,question)\n",
    "#         new_datas.append({\"instruction\":instruction,\"output\":answer})\n",
    "        inputs.append(instruction)\n",
    "        outputs.append(answer)\n",
    "    except:\n",
    "        ..."
   ],
   "id": "a90f5974cbfcaf79"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "len(inputs),len(outputs)",
   "id": "40f96cbb3f6b3c1f"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "new_datasets = Dataset.from_dict({\"instruction\":inputs,\"output\":outputs})",
   "id": "b8c7fbc3395e2c38"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "new_datasets.save_to_disk(\"qa_datasets\")",
   "id": "85d19f20dba3ad1"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "len(new_datas)",
   "id": "a5ce9907db205850"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "new_datas[0]",
   "id": "186d4f0e004656e8"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "from datasets import load_from_disk",
   "id": "ec62e8c794d636bd"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "load_from_disk(\"qa_datasets\").shuffle(seed=42)",
   "id": "e5d19c7cb4cfbdb1"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "ks=['computer_network', 'operating_system', 'computer_architecture', 'college_programming', 'college_physics', 'college_chemistry', 'advanced_mathematics', 'probability_and_statistics', 'discrete_mathematics', 'electrical_engineer', 'metrology_engineer', 'high_school_mathematics', 'high_school_physics', 'high_school_chemistry', 'high_school_biology', 'middle_school_mathematics', 'middle_school_biology', 'middle_school_physics', 'middle_school_chemistry', 'veterinary_medicine', 'college_economics', 'business_administration', 'marxism', 'mao_zedong_thought', 'education_science', 'teacher_qualification', 'high_school_politics', 'high_school_geography', 'middle_school_politics', 'middle_school_geography', 'modern_chinese_history', 'ideological_and_moral_cultivation', 'logic', 'law', 'chinese_language_and_literature', 'art_studies', 'professional_tour_guide', 'legal_professional', 'high_school_chinese', 'high_school_history', 'middle_school_history', 'civil_servant', 'sports_science', 'plant_protection', 'basic_medicine', 'clinical_medicine', 'urban_and_rural_planner', 'accountant', 'fire_engineer', 'environmental_impact_assessment_engineer', 'tax_accountant', 'physician']\n",
    "prompts_up=[\"请从选项中选择一个最佳答案回答这个问题：{}\\n{}\",\n",
    "        \"从以下几个答案中选择一个作答：{}\\n{}\",\n",
    "        \"A、B、C、D选项中哪个是这个问题的答案：{}\\n{}\",\n",
    "        \"A、B、C、D中哪个是答案：{}\\n{}\",\n",
    "           \"{}\\n{}\"]\n",
    "prompts_down=[\"{}\\n请从选项中选择一个最佳答案回答这个问题\\n{}\",\n",
    "        \"{}\\n从以下几个答案中选择一个作答\\n{}\",\n",
    "        \"{}\\nA、B、C、D选项中哪个是这个问题的答案\\n{}\",\n",
    "        \"{}\\nA、B、C、D中哪个是答案\\n{}\",\n",
    "             \"{}\\n{}\"]\n",
    "inputs,outputs=[],[]\n",
    "for k in ks:\n",
    "    ds1=load_dataset(r\"ceval/ceval-exam\",name=k)\n",
    "    for _k in ['test','val','dev']:\n",
    "        for data in ds1[_k]:\n",
    "            question,abcd=data['question'],f\"A:{data['A']}\\nB:{data['B']}\\nC:{data['C']}\\nD:{data['D']}\"\n",
    "            if random.random()>=0.5:\n",
    "                prompt=random.choice(prompts_up)\n",
    "            else:\n",
    "                prompt=random.choice(prompts_down)\n",
    "            instruction=prompt.format(question,abcd)\n",
    "            inputs.append(instruction)\n",
    "            outputs.append(data['answer'])"
   ],
   "id": "ff4cee3a934e3669"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds1['test'][0]",
   "id": "28bc734b9114053"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds1",
   "id": "da4a322cc922366c"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "len(inputs),len(outputs)",
   "id": "41f78eba74d2b437"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "new_datasets = Dataset.from_dict({\"instruction\":inputs,\"output\":outputs})",
   "id": "d7e2180be7725ea1"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "new_datasets",
   "id": "610e04834674c9d0"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "new_datasets.save_to_disk(\"ceval\")",
   "id": "7a29905bbd780f21"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds=load_dataset('json', data_files=f'zip/LLMZoo/llmzoo/eval/questions/questions-zh.jsonl')",
   "id": "8ac2189345b813db"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds",
   "id": "535839cfaa62e261"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "id2q={data['question_id']:data['text'] for data in ds['train']}",
   "id": "6f3111c4be0f40a8"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds=load_dataset('json', data_files=f'zip/LLMZoo/llmzoo/eval/answers/answer_gpt35.jsonl')",
   "id": "66756bc6e2066de"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "id2a={data['question_id']:data['text'] for data in ds['train']}",
   "id": "8de1774451e3958d"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "inputs,outputs=[],[]\n",
    "for k in id2q:\n",
    "    inputs.append(id2q[k])\n",
    "    outputs.append(id2a[k])\n",
    "new_datasets = Dataset.from_dict({\"instruction\":inputs,\"output\":outputs})\n",
    "new_datasets.save_to_disk(\"llmzoo\")"
   ],
   "id": "708163084cb07fe0"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "# ds1=load_dataset(\"fnlp/moss-002-sft-data\",split=\"zh_helpfulness\")\n",
    "# ds2=load_dataset(\"fnlp/moss-002-sft-data\",split=\"zh_honesty\")"
   ],
   "id": "d02d199a24e863ae"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "import os\n",
    "import json\n",
    "import random\n",
    "from datasets import load_dataset,concatenate_datasets,Dataset\n",
    "from tqdm import tqdm"
   ],
   "id": "ea238efdb2e5833d"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "ds=load_dataset(\"fnlp/moss-002-sft-data\",split=\"zh_helpfulness.json\")",
   "id": "962f28e1acf7c18b"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "from huggingface_hub import snapshot_download\n",
    "snapshot_download(repo_id=\"nlp/moss-002-sft-data\")"
   ],
   "id": "f572fc5bba9e1621"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "dc911917c09698c4"
  }
 ],
 "metadata": {},
 "nbformat": 5,
 "nbformat_minor": 9
}
